package chapter03

import org.apache.spark.{SparkConf, SparkContext}

object Test19_filterSalary {
  def main(args: Array[String]): Unit = {
    val con = new SparkConf().setMaster("local[*]").setAppName("filterSalary")
    val sc = new SparkContext(con)
    val value = sc.textFile("input/Employee_salary_first_half.csv")
    val value1 = sc.textFile("input/Employee_salary_second_half.csv")
    //去掉表头
    val value2 = value.mapPartitionsWithIndex((index, e) => {
      if (index == 0) e.drop(1) else e
    })
    val value3 = value1.mapPartitionsWithIndex((index, e) => {
      if (index == 0) e.drop(1) else e
    })
    //过滤工资数
    val value4 = value2.map(e => e.split(","))
      .map(e => (e(1), e(5).toInt,e(9)))
      .filter(e => e._2 > 20000)
    val value5 = value3.map(e => e.split(","))
      .map(e => (e(1), e(5).toInt,e(9)))
      .filter(e => e._2 > 20000)
    //合并数据
    val value6 = value4.union(value5)
//      .map(e => e._1)
      .distinct()
    println(value6.collect().toList)

  }
}
